package sis.ra.relationExtraction;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.TreeSet;

import sis.ra.partofspeech.AlliasAnalysis;
import sis.ra.partofspeech.wordindex;
import sis.ra.utility.InxEntity;
import sis.ra.utility.Utils;


public class parseYahooResults {

	public static HashMap<String, ArrayList<String>> getYahooResult()
	{
		String path="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\yahoo result_922\\";
		File filepath=new File(path);
		String [] files=filepath.list();
		HashMap<String, ArrayList<String>> yahooresult=new HashMap<String, ArrayList<String>> ();
		for (int i=0;i<files.length;i++)
		{
			String filename=path+files[i].toString();
			if (filename.contains(".result")) continue;
			String filecontent=Utils.readFile(filename);
			String companyname=files[i].toString().replace(".txt", "");
			
			ArrayList<String> products=new ArrayList<String> ();
			String [] lines=filecontent.split("\n");
			int t=0;
			int prelength=0;
			String thisline="";
			for (int j=0;j<lines.length;j++)
			{
				if (lines[j].startsWith("WEB RESULTS")) continue;
				t=t+1;
				if (t==2) 
				{
					prelength=lines[j].length();
					thisline=lines[j];
				}
				if (t==3) 
				{
					if (lines[j].length()>prelength)
					{
						thisline=lines[j];
						prelength=lines[j].length();
						t=2;
					}else
					t=0;
					if (!products.contains(thisline))
						products.add(thisline);
				}
			}
			yahooresult.put(companyname, products);
	//		System.out.println(companyname+" "+products.size());
		}//end of files
		
		return yahooresult;
	}
	
	public static HashMap<String, ArrayList<String>> getCompanyProducts()
	{
		String file="C:\\Documents and Settings\\I820753\\Desktop\\dataset\\products.txt";
		String content=Utils.readFile(file);
//		System.out.println(content);
		String [] lines=content.split("\n");
		HashMap<String, ArrayList<String>> yahooresult=new HashMap<String, ArrayList<String>> ();
		
		for (int i=0;i<lines.length;i++)
		{
			String [] items=lines[i].split("\t");
			
			String company=items[0];
			ArrayList<String> productslist=new ArrayList<String> ();
			String products="";
			if (items.length>2) products=items[2];
		//	System.out.println(company+" "+products);
			if (products.contains(";"))
			{
				String[] itempro=products.split(" ; ");
				for (int j=0;j<itempro.length;j++)
				{
					if (itempro[j].length()<1) continue;
					if (productslist.contains(itempro[j])) continue;
					productslist.add(itempro[j]);
	//				System.out.println(itempro[j]);
				}
			}
	//		System.out.println(company);
			yahooresult.put(company, productslist);
		}
		return yahooresult;
	}
	
	public static void stem (String text, String htformat)
	{
		String linguistxpath="c:\\Program Files\\INXIGHT\\LinguistxPlatform3.7\\lx-3\\";

		Utils.writeFile(linguistxpath+"windows-2003-i686-msvc-7.1-32\\test.txt ", text);
		String s="";
		
		String cmd="\""+linguistxpath+"windows-2003-i686-msvc-7.1-32\\test_platform.exe\" -k 735442535d524e "+
		" -stem " +
		" -"+htformat+" " +"\"" +linguistxpath + "windows-2003-i686-msvc-7.1-32\\test.txt\" ";
		try {
			System.out.println(cmd);
		     Process p = Runtime.getRuntime().exec(cmd);
		    
		    BufferedReader stdInput = new BufferedReader(new InputStreamReader(p.getInputStream()));
		    BufferedReader stdError = new BufferedReader(new InputStreamReader(p.getErrorStream()));
		    while ((s = stdInput.readLine()) != null)   
		    {
		    	System.out.println(s);		    
		    }
		}
		catch (IOException e) {
			    e.printStackTrace();    
		}
	}
	
	public static HashMap<Integer, wordindex> pos (String text, String htformat)
	{
		String linguistxpath="c:\\Program Files\\INXIGHT\\LinguistxPlatform3.7\\lx-3\\";
		String posresult="";
		Utils.writeFile(linguistxpath+"windows-2003-i686-msvc-7.1-32\\test.txt ", text);
		String s="";
		String encoding="";
		String format="";
		String language="";
		String sentencestart="";
		String sentenceend="";
		int start=0;
		int end=0;
		int offset=0;
		Integer count=0;
		HashMap<Integer,wordindex> sentenceindex=new HashMap<Integer, wordindex> ();
		ArrayList<wordindex> sentenceindexArray=new ArrayList<wordindex> ();
		ArrayList<wordindex> posindexArray=new ArrayList<wordindex> ();
		
		String cmd="\""+linguistxpath+"windows-2003-i686-msvc-7.1-32\\test_platform.exe\" -k 735442535d524e "+
//		" -tag " +
		" -tokenize " + 
		"-d " + "\""+linguistxpath + "lang\"" + 
//		" -stem " +
		" -"+htformat+" " +"\"" +linguistxpath + "windows-2003-i686-msvc-7.1-32\\test.txt\" ";
		try {
//			System.out.println(cmd);
		     Process p = Runtime.getRuntime().exec(cmd);
		    
		    BufferedReader stdInput = new BufferedReader(new InputStreamReader(p.getInputStream()));
		    BufferedReader stdError = new BufferedReader(new InputStreamReader(p.getErrorStream()));
		    while ((s = stdInput.readLine()) != null)   
		    {
	//	    	System.out.println(s);
		    	// analyze token part
		    	if (s.contains("segment tokenization -- tokens:"))
		    	{
		    		while (!(s= stdInput.readLine()).contains("segment tokenization -- normalized tokens:"))
		    		{
		    			if (s.startsWith("  ")&&s.contains("start=")&&s.contains("end=")&&!s.startsWith("     "))
		    				{
		//    					System.out.println(s);
		    					String [] t=s.split("\t");
		    					if (t.length<4) continue;
		    					//wordindex(String pos, Integer position, String token, String type, String word)
		    					wordindex wi=new wordindex("", Integer.parseInt(t[2].substring(6).trim()),"","", t[0].trim());
		    					sentenceindex.put(Integer.parseInt(t[2].substring(6).trim()), wi);
		    					sentenceindexArray.add(wi);
		    				}
		    		}
		    	}
		    	
		    	if (s.contains("segment tokenization -- tags:"))
		    	{
		    		
		    		while ((s= stdInput.readLine())!=null&&!s.contains("The character encoding of this segment is"))
		    		{
		    			
		    			if (s.startsWith("  ")&&s.trim().length()>2&&!s.startsWith("      "))
		    			{
	//	    				System.out.println("=="+s);
		    				String [] t=s.split("\t");
		    				if (t.length>2) continue;
		    				//public wordindex(String entity, String entitytype, String pos,Integer position, String token, String type, String word) {
	//	    				System.out.println(count+" "+sentenceindexArray.get(count).getWord());
		    				sentenceindexArray.get(count).setPos(t[1].trim());
		    				
		    				count=count+1;
		    			}		    			
		    		}
		    	}
		    }
	//	    while ((s = stdError.readLine()) != null)   System.out.println(s);
		}
		catch (IOException e) {
			    e.printStackTrace();    
		}

		return sentenceindex;
	}
	
	public static HashMap<Integer, wordindex> posNeAlias(String contents)
	{
		String ne=Utils.getNE(contents);
		HashMap<Integer,InxEntity> nes=Utils.parseIMSXMLContentHash(ne);
		HashMap<Integer,String> aliasindex=AlliasAnalysis.aliasAnalysis(ne);
		TreeSet<Integer> sortedSetne=new TreeSet<Integer>(nes.keySet());
		Object [] sortedArrayne=sortedSetne.toArray();
		HashMap<Integer,wordindex> posresult=pos(contents, "text");
		TreeSet<Integer> sortedSet=new TreeSet<Integer>(posresult.keySet());
		Object [] sortedArray=sortedSet.toArray();
		for (int i=0;i<sortedArrayne.length;i++)
		{
			Integer neindexnow=(Integer)sortedArrayne[i];
			Integer posindexnow=neindexnow;
			if (posresult.containsKey(posindexnow)&&nes.get(neindexnow).getSurface().startsWith(posresult.get(posindexnow).getWord()) ) 
			{
				
//				System.out.println(neindexnow+" "+nes.get(neindexnow).getEntityType()+" "+nes.get(neindexnow).getSurface());
				posresult.get(posindexnow).setEntitytype(nes.get(neindexnow).getEntityType());
				posresult.get(posindexnow).setEntity(nes.get(neindexnow).getSurface());	
				if (aliasindex.containsKey(neindexnow))
					posresult.get(posindexnow).setReference(aliasindex.get(neindexnow).split("\t")[1]);
				//System.out.println(neindexnow+" "+nes.get(neindexnow).getEntityType()+" "+nes.get(neindexnow).getSurface());
			}else
			{
				posindexnow=posindexnow-10;
				int loop=0;
				while (!posresult.containsKey(posindexnow)||!nes.get(neindexnow).getSurface().toLowerCase().trim().contains(posresult.get(posindexnow).getWord().toLowerCase().trim()))
				{
					posindexnow=posindexnow+1;
//					System.out.println("=="+posindexnow+" "+nes.get(neindexnow).getSurface().toLowerCase().trim());
					loop=loop+1;
					if (loop>20) break;					
				}
				if (loop>20) 
				{
					neindexnow=(Integer)sortedArrayne[i];
//					posresult.get(neindexnow).setEntitytype(nes.get(neindexnow).getEntityType());
//					posresult.get(neindexnow).setEntity(nes.get(neindexnow).getSurface());		
				}else{
				posresult.get(posindexnow).setEntitytype(nes.get(neindexnow).getEntityType());
				posresult.get(posindexnow).setEntity(nes.get(neindexnow).getSurface());	
				if (aliasindex.containsKey(neindexnow))
					posresult.get(posindexnow).setReference(aliasindex.get(neindexnow).split("\t")[1]);
				}
			}
		}
		return posresult;
	}
	
	public static String toString(HashMap<Integer,wordindex> posresult)
	{
		
		TreeSet<Integer> sortedSet=new TreeSet<Integer>(posresult.keySet());
		Object [] sortedArray=sortedSet.toArray();

		//output the wordindexresult
		String out="";
		for (int i=0;i<sortedArray.length;i++)
		{
			if (posresult.get(sortedArray[i]).getEntity()==null)
				out=out+sortedArray[i]+"\t"+posresult.get(sortedArray[i]).getWord()+"\t"+posresult.get(sortedArray[i]).getPos()+"\n";
			else
			{
				if (posresult.get(sortedArray[i]).getReference()==null)
				out=out+sortedArray[i]+"\t"+posresult.get(sortedArray[i]).getWord()+"\t"+posresult.get(sortedArray[i]).getPos()+"\t"+
					posresult.get(sortedArray[i]).getEntity()+"\t"+posresult.get(sortedArray[i]).getEntitytype()+"\n";		
				else
					out=out+sortedArray[i]+"\t"+posresult.get(sortedArray[i]).getWord()+"\t"+posresult.get(sortedArray[i]).getPos()+"\t"+
					posresult.get(sortedArray[i]).getEntity()+"\t"+posresult.get(sortedArray[i]).getEntitytype()+"\t"+posresult.get(sortedArray[i]).getReference()+"\n";		
			}
		}
		return out;
	}
	
	public static ArrayList <String> getPattern(ArrayList <String> sentence, ArrayList <String> products)
	{
		ArrayList <String> pattern=new ArrayList<String>();
		String companyname="";
		for (int i=0;i<sentence.size();i++)
		{
			if (sentence.get(i).contains("COMPANY"))
			{
				pattern.add("COMPANY");
				companyname=sentence.get(i).split("\t")[3];
	//			System.out.println(companyname);
			}
			else{
				boolean isProducts=false;
				for (int j=0;j<products.size();j++)
				{
					if (sentence.get(i).contains(products.get(j).trim().toLowerCase())&&!sentence.get(i).contains("COMPANY"))
					{
						pattern.add("PRODUCTS");
						isProducts=true;
					}
				}
				if (isProducts) pattern.add("PRODUCTS");
				else 
				{
					String t=sentence.get(i).split("\t")[1];
					String t1=sentence.get(i).split("\t")[2];
					if (companyname.contains(t)) continue;
					if (t1.contains("V-Pres-3-Sg")||t1.contains("V-Pres-Sg-be"))
						pattern.add(t);
					else
					pattern.add(sentence.get(i));
				}
			}
				
		}
		return pattern;
	}
	
	public static ArrayList<String> matchPattern(ArrayList <String> sentence, ArrayList <String> products)
	{
		ArrayList<String> potentialPattern=new ArrayList<String>();
		boolean containCompany=false;
		for (int i=0;i<sentence.size();i++)
		{
	//		System.out.println(sentence.get(i));
			if (sentence.get(i).contains("COMPANY"))
			{
				containCompany=true;
		//		System.out.println(products);
				break;
			}
		}
		
		boolean containProducts=false;
		
		if (containCompany&&products!=null) 
		{
			for (int i=0;i<sentence.size();i++)
			{
	//			System.out.println(sentence.get(i));
				for (int j=0;j<products.size();j++)
				{
	//				System.out.println(sentence.get(i)+" "+products.get(j).trim());
					if (sentence.get(i).contains(products.get(j).trim().toLowerCase())&&!sentence.get(i).contains("COMPANY"))
					{
	//					System.out.println(sentence+" "+products.get(j));
						containProducts=true;
						break;
					}
				}
			}
		}
		
		if (containProducts)
		{
			potentialPattern=getPattern(sentence,products);
		}
		return potentialPattern;
	}
	
	public static void main(String[] args) {
		HashMap<String, ArrayList<String>> yahooresult=getYahooResult();
		HashMap<String, ArrayList<String>> companyproducts=getCompanyProducts();
		Iterator<String> companynames=yahooresult.keySet().iterator();
		String results="";
		String output="";
		while (companynames.hasNext())
		{
			String companyname=companynames.next();
			results=results+companyname+"\n";
//			if (!companyname.equals("ExxonMobil")) continue;
			ArrayList<String> products=companyproducts.get(companyname);
			ArrayList<String> yahooresults=yahooresult.get(companyname);
			output=output+companyname+"\n";
			for (int i=0;i<yahooresults.size();i++)
			{				
				String result=yahooresults.get(i);
	//			System.out.println(result);
				results=results+result+"\n";
				
	//			String test="Get Marathon Oil Corp. (MRO) stock research & investing information. Find historical stock quotes, key competitors, stock ... and China Petrochemical Corp ...";
				output=output+result+"\n";
				HashMap<Integer,wordindex> posresult=posNeAlias(result);
				
				TreeSet<Integer> sortedIndex=new TreeSet<Integer>(posresult.keySet());
				Object [] sortedArray=sortedIndex.toArray();

				ArrayList<String> sentence=new ArrayList<String> ();				
				String out="";
				for (int j=0;j<sortedArray.length;j++)
				{
					/*
					 * using "." and "..." to seperate the sentence
					 * Potiential sentence snippet put together and 
					 * analysis in getPattern
					 */
					
	//				System.out.println(posresult.get(sortedArray[j]).getWord());
					if (!posresult.get(sortedArray[j]).getWord().equals(".")&&!posresult.get(sortedArray[j]).getWord().equals("..."))
					{
						if (posresult.get(sortedArray[j]).getEntity()==null)
							out=sortedArray[j]+"\t"+posresult.get(sortedArray[j]).getWord()+"\t"+posresult.get(sortedArray[j]).getPos()+"\n";
						else
						{
							if (posresult.get(sortedArray[j]).getReference()==null)
							out=sortedArray[j]+"\t"+posresult.get(sortedArray[j]).getWord()+"\t"+posresult.get(sortedArray[j]).getPos()+"\t"+
								posresult.get(sortedArray[j]).getEntity()+"\t"+posresult.get(sortedArray[j]).getEntitytype()+"\n";		
							else
								out=sortedArray[j]+"\t"+posresult.get(sortedArray[j]).getWord()+"\t"+posresult.get(sortedArray[j]).getPos()+"\t"+
								posresult.get(sortedArray[j]).getEntity()+"\t"+posresult.get(sortedArray[j]).getEntitytype()+"\t"+posresult.get(sortedArray[j]).getReference()+"\n";		
						}
						sentence.add(out);
					}else
					{
						ArrayList<String> potientialpattern=matchPattern(sentence, products);
						if (potientialpattern.size()>0)
						output=output+potientialpattern+"\n";
						sentence=new ArrayList<String> ();
					}
				}
			}			
		}//while
		Utils.writeFile("C:\\Documents and Settings\\I820753\\Desktop\\dataset\\yahoo result_922\\pattern.result", output);
		System.out.println(output);
	}
}
